{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# An RNN for short-term predictions\n",
    "This model will try to predict the next value in a short sequence based on historical data. This can be used for example to forecast demand based on a couple of weeks of sales data.\n",
    "\n",
    "<div class=\"alert alert-block alert-info\">\n",
    "This version packages the model into an Estimator interface.\n",
    "It can be executed on Google cloud ML Engine using the code in this bash notebook: [../run-on-cloud-ml-engine.ipynb](../run-on-cloud-ml-engine.ipynb)\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.platform import tf_logging as logging\n",
    "logging.set_verbosity(logging.INFO)\n",
    "logging.log(logging.INFO, \"Tensorflow version \" + tf.__version__)\n",
    "import utils_datagen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "display"
    ]
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import utils_display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate fake dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_SEQ_LEN = 1024*128\n",
    "data = np.concatenate([utils_datagen.create_time_series(waveform, DATA_SEQ_LEN) for waveform in utils_datagen.Waveforms])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "display"
    ]
   },
   "outputs": [],
   "source": [
    "utils_display.picture_this_1(data, DATA_SEQ_LEN)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RNN_CELLSIZE = 32    # size of the RNN cells\n",
    "SEQLEN = 16         # unrolled sequence length\n",
    "BATCHSIZE = 32      # mini-batch size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize training sequences\n",
    "This is what the neural network will see during training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "display"
    ]
   },
   "outputs": [],
   "source": [
    "utils_display.picture_this_2(data, BATCHSIZE, SEQLEN) # execute multiple times to see different sample sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The model definition\n",
    "When executed, this function instantiates the Tensorflow graph for our model.\n",
    "![deep RNN schematic](../../../docs/images/RNN1.svg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tree simplistic predictive models: can you beat them ?\n",
    "def simplistic_models(X):\n",
    "    # \"random\" model\n",
    "    Yrnd = tf.random_uniform([tf.shape(X)[0]], -2.0, 2.0) # tf.shape(X)[0] is the batch size\n",
    "    # \"same as last\" model\n",
    "    Ysal = X[:,-1]\n",
    "    # \"trend from last two\" model\n",
    "    Ytfl = X[:,-1] + (X[:,-1] - X[:,-2])\n",
    "    return Yrnd, Ysal, Ytfl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bad_model(X):\n",
    "    Yr = X * tf.Variable(tf.ones([]), name=\"dummy\") # shape [BATCHSIZE, SEQLEN]\n",
    "    Yout = Yr[:,-1:SEQLEN] # Last item in sequence. Yout [BATCHSIZE, 1]\n",
    "    return Yout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# linear model (RMSE: 0.36, with shuffling: 0.17)\n",
    "def linear_model(X):\n",
    "    Yout = tf.layers.dense(X, 1) # output shape [BATCHSIZE, 1]\n",
    "    return Yout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2-layer dense model (RMSE: 0.38, with shuffling: 0.15-0.18)\n",
    "def DNN_model(X):\n",
    "    Y = tf.layers.dense(X, SEQLEN//2, activation=tf.nn.relu)\n",
    "    Yout = tf.layers.dense(Y, 1, activation=None) # output shape [BATCHSIZE, 1]\n",
    "    return Yout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convolutional (RMSE: 0.31, with shuffling: 0.16)\n",
    "def CNN_model(X):\n",
    "    X = tf.expand_dims(X, axis=2) # [BATCHSIZE, SEQLEN, 1] is necessary for conv model\n",
    "    Y = tf.layers.conv1d(X, filters=8, kernel_size=4, activation=tf.nn.relu, padding=\"same\") # [BATCHSIZE, SEQLEN, 8]\n",
    "    Y = tf.layers.conv1d(Y, filters=16, kernel_size=3, activation=tf.nn.relu, padding=\"same\") # [BATCHSIZE, SEQLEN, 8]\n",
    "    Y = tf.layers.conv1d(Y, filters=8, kernel_size=1, activation=tf.nn.relu, padding=\"same\") # [BATCHSIZE, SEQLEN, 8]\n",
    "    Y = tf.layers.max_pooling1d(Y, pool_size=2, strides=2)  # [BATCHSIZE, SEQLEN//2, 8]\n",
    "    Y = tf.layers.conv1d(Y, filters=8, kernel_size=3, activation=tf.nn.relu, padding=\"same\")  # [BATCHSIZE, SEQLEN//2, 8]\n",
    "    Y = tf.layers.max_pooling1d(Y, pool_size=2, strides=2)  # [BATCHSIZE, SEQLEN//4, 8]\n",
    "    # mis-using a conv layer as linear regression :-)\n",
    "    Yout = tf.layers.conv1d(Y, filters=1, kernel_size=SEQLEN//4, activation=None, padding=\"valid\") # output shape [BATCHSIZE, 1, 1]\n",
    "    Yout = tf.squeeze(Yout, axis=-1) # output shape [BATCHSIZE, 1]\n",
    "    return Yout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RNN model (RMSE: 0.38, with shuffling 0.14, the same with loss on last 8)\n",
    "def RNN_model(X, n=1):\n",
    "    # 2-layer RNN\n",
    "    X = tf.expand_dims(X, axis=2) # [BATCHSIZE, SEQLEN, 1] is necessary for RNN model\n",
    "    cell1 = tf.nn.rnn_cell.GRUCell(RNN_CELLSIZE)\n",
    "    cell2 = tf.nn.rnn_cell.GRUCell(RNN_CELLSIZE)\n",
    "    cell = tf.nn.rnn_cell.MultiRNNCell([cell1, cell2], state_is_tuple=False)\n",
    "    Yn, H = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32) # Yn [BATCHSIZE, SEQLEN, RNN_CELLSIZE]\n",
    "    \n",
    "    # regression head\n",
    "    batchsize = tf.shape(X)[0]\n",
    "    Yn = tf.reshape(Yn, [batchsize*SEQLEN, RNN_CELLSIZE])\n",
    "    Yr = tf.layers.dense(Yn, 1) # Yr [BATCHSIZE*SEQLEN, 1]\n",
    "    Yr = tf.reshape(Yr, [batchsize, SEQLEN, 1]) # Yr [BATCHSIZE, SEQLEN, 1]\n",
    "    \n",
    "    # In this RNN model, you can compute the loss on the last predicted item or the lats n predicted items\n",
    "    # Last n is slightly better.\n",
    "    Yout = Yr[:,-n:SEQLEN,:] # last item(s) in sequence: output shape [BATCHSIZE, n, 1]\n",
    "    Yout = tf.squeeze(Yout, axis=-1)\n",
    "    return Yout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RNN_model_N(X): return RNN_model(X, n=SEQLEN//2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def configurable_model_fn(features, labels, mode, model=CNN_model):\n",
    "    X = features # shape [BATCHSIZE, SEQLEN]\n",
    "    Y = model(X)\n",
    "    Yout = Y[:,-1]\n",
    "    \n",
    "    loss = train_op = eval_metrics = None\n",
    "    if mode != tf.estimator.ModeKeys.PREDICT:\n",
    "        last_label = labels[:, -1] # last item in sequence: the target value to predict\n",
    "        last_labels = labels[:, -tf.shape(Y)[1]:SEQLEN] # last p items in sequence (as many as in Y), useful for RNN_model(X, n>1)\n",
    "\n",
    "        loss = tf.losses.mean_squared_error(Y, last_labels) # loss computed on last label(s)\n",
    "        optimizer = tf.train.AdamOptimizer(learning_rate=0.01)\n",
    "        train_op = tf.contrib.training.create_train_op(loss, optimizer)\n",
    "        #train_op = optimizer.minimize(loss)\n",
    "        Yrnd, Ysal, Ytfl = simplistic_models(X)\n",
    "        eval_metrics = {\"RMSE\": tf.metrics.root_mean_squared_error(Y, last_labels),\n",
    "                        # compare agains three simplistic predictive models: can you beat them ?\n",
    "                        \"RMSErnd\": tf.metrics.root_mean_squared_error(Yrnd, last_label),\n",
    "                        \"RMSEsal\": tf.metrics.root_mean_squared_error(Ysal, last_label),\n",
    "                        \"RMSEtfl\": tf.metrics.root_mean_squared_error(Ytfl, last_label)}\n",
    "\n",
    "    return tf.estimator.EstimatorSpec(\n",
    "        mode = mode,\n",
    "        predictions = {\"Yout\":Yout},\n",
    "        loss = loss,\n",
    "        train_op = train_op,\n",
    "        eval_metric_ops = eval_metrics\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode): return configurable_model_fn(features, labels, mode, model=RNN_model_N)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# prepare training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training to predict the same sequence shifted by one (next value)\n",
    "labeldata = np.roll(data, -1)\n",
    "# slice data into sequences\n",
    "traindata = np.reshape(data, [-1, SEQLEN])\n",
    "labeldata = np.reshape(labeldata, [-1, SEQLEN])\n",
    "\n",
    "# also make an evaluation dataset by randomly subsampling our fake data\n",
    "EVAL_SEQUENCES = DATA_SEQ_LEN*4//SEQLEN//4\n",
    "joined_data = np.stack([traindata, labeldata], axis=1) # new shape is [N_sequences, 2(train/eval), SEQLEN]\n",
    "joined_evaldata = joined_data[np.random.choice(joined_data.shape[0], EVAL_SEQUENCES, replace=False)]\n",
    "evaldata = joined_evaldata[:,0,:]\n",
    "evallabels = joined_evaldata[:,1,:]\n",
    "\n",
    "def train_dataset():\n",
    "    # Dataset API for batching, shuffling, repeating\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((traindata, labeldata))\n",
    "    dataset = dataset.repeat() # indefinitely\n",
    "    dataset = dataset.shuffle(DATA_SEQ_LEN*4//SEQLEN) # important ! Number of sequences in shuffle buffer: all of them\n",
    "    dataset = dataset.batch(BATCHSIZE)\n",
    "    \n",
    "    samples, labels = dataset.make_one_shot_iterator().get_next()\n",
    "    return samples, labels\n",
    "\n",
    "def eval_dataset():\n",
    "    # Dataset API for batching\n",
    "    evaldataset = tf.data.Dataset.from_tensor_slices((evaldata, evallabels))\n",
    "    evaldataset = evaldataset.repeat(1)\n",
    "    evaldataset = evaldataset.batch(EVAL_SEQUENCES) # just one batch with everything\n",
    "\n",
    "    samples, labels = evaldataset.make_one_shot_iterator().get_next()\n",
    "    return samples, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_EPOCHS = 5\n",
    "nbsteps = DATA_SEQ_LEN*4//SEQLEN//BATCHSIZE * NB_EPOCHS\n",
    "train_spec = tf.estimator.TrainSpec(input_fn=train_dataset, max_steps=nbsteps)\n",
    "eval_spec = tf.estimator.EvalSpec(input_fn=eval_dataset, throttle_secs=20, start_delay_secs=1, steps=None) # eval until dataset exhausted (1 batch)\n",
    "training_config = tf.estimator.RunConfig(model_dir=\"./outputdir\")\n",
    "estimator=tf.estimator.Estimator(model_fn=model_fn, config=training_config)\n",
    "tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "display"
    ]
   },
   "outputs": [],
   "source": [
    "evals = estimator.evaluate(eval_dataset)\n",
    "results = estimator.predict(eval_dataset)\n",
    "Yout_ = [result[\"Yout\"] for result in results]\n",
    "utils_display.picture_this_3(Yout_, evaldata, evallabels, SEQLEN) # execute multiple times to see different sample sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright 2018 Google LLC\n",
    "\n",
    "Licensed under the Apache License, Version 2.0 (the \"License\");\n",
    "you may not use this file except in compliance with the License.\n",
    "You may obtain a copy of the License at\n",
    "[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)\n",
    "Unless required by applicable law or agreed to in writing, software\n",
    "distributed under the License is distributed on an \"AS IS\" BASIS,\n",
    "WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
    "See the License for the specific language governing permissions and\n",
    "limitations under the License."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
